{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SMS Spam Classification\n",
    "This notebook illustrates classification of SMS as SPAM or NOT SPAM using Recurrent Neural Network and LSTM."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>v1</th>\n",
       "      <th>v2</th>\n",
       "      <th>Unnamed: 2</th>\n",
       "      <th>Unnamed: 3</th>\n",
       "      <th>Unnamed: 4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     v1                                                 v2 Unnamed: 2  \\\n",
       "0   ham  Go until jurong point, crazy.. Available only ...        NaN   \n",
       "1   ham                      Ok lar... Joking wif u oni...        NaN   \n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...        NaN   \n",
       "3   ham  U dun say so early hor... U c already then say...        NaN   \n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...        NaN   \n",
       "\n",
       "  Unnamed: 3 Unnamed: 4  \n",
       "0        NaN        NaN  \n",
       "1        NaN        NaN  \n",
       "2        NaN        NaN  \n",
       "3        NaN        NaN  \n",
       "4        NaN        NaN  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('data/spam.csv',encoding='latin-1')\n",
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = data.rename(columns={\"v2\" : \"text\", \"v1\":\"label\"})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing\n",
    "Lets save our labels and messages to text files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.savetxt(r'data\\messages.txt', data['text'].values, fmt='%s')\n",
    "np.savetxt(r'data\\labels.txt', data['label'].values, fmt='%s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('data/messages.txt', encoding=\"ISO-8859-1\") as f:\n",
    "    messages = f.read()\n",
    "with open('data/labels.txt',encoding=\"ISO-8859-1\") as f:\n",
    "    labels = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...\\nOk lar... Joking wif u oni...\\nFree entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005. Text FA to 87121 to receive entry question(std txt rate)T&C's apply 08452810075over18's\\nU dun say so early hor... U c already then say...\\nNah I don't think he goes to usf, he lives around here though\\nFreeMsg Hey there darling it's been 3 week's now and no word back! I'd like some fun you u\""
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "messages[:500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ham\\nham\\nspam\\nham\\nham\\nspam\\nham\\nham\\nspam\\nspam\\nham\\nspam\\nspam\\nham\\nham\\nspam\\nham\\nham\\nham\\nspam\\nham\\nham\\nham\\n'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels[:100]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Remove punctuations such a (. , !) etc and seperate using delimiter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from string import punctuation\n",
    "all_text = ''.join([c for c in messages if c not in punctuation])\n",
    "messages = all_text.split('\\n')\n",
    "\n",
    "all_text = ' '.join(messages)\n",
    "words = all_text.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go until jurong point crazy Available only in bugis n great world la e buffet Cine there got amore wat Ok lar Joking wif u oni Free entry in 2 a wkly comp to win FA Cup final tkts 21st May 2005 Text FA to 87121 to receive entry questionstd txt rateTCs apply 08452810075over18s U dun say so early hor U c already then say Nah I dont think he goes to usf he lives around here though FreeMsg Hey there darling its been 3 weeks now and no word back Id like some fun you up for it still Tb ok XxX std chgs\n",
      "\n",
      "\n",
      "['Go', 'until', 'jurong', 'point', 'crazy', 'Available', 'only', 'in', 'bugis', 'n', 'great', 'world', 'la', 'e', 'buffet', 'Cine', 'there', 'got', 'amore', 'wat']\n"
     ]
    }
   ],
   "source": [
    "print (all_text[:500])\n",
    "print (\"\\n\")\n",
    "print (words[:20])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building our vocabulary and converting messages to vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "split_words = Counter(words)\n",
    "sorted_split_words = sorted(split_words, key=split_words.get, reverse=True)\n",
    "vocab_to_int = {c : i for i, c in enumerate(sorted_split_words,1)}\n",
    "\n",
    "# Convert the reviews to integers, same shape as reviews list, but with integers\n",
    "messages_ints = []\n",
    "for message in messages:\n",
    "    messages_ints.append([vocab_to_int[i] for i in message.split()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['to', 'you', 'I', 'a', 'the', 'and', 'in', 'is', 'u', 'i', 'me', 'for', 'my', 'of', 'your', 'it', 'on', '2', 'have', 'that', 'are', 'call', 'now', 'or', 'be', 'not', 'at', 'with', 'U', 'get', 'will', 'can', 'Im', 'so', 'ur', '4', 'but', 'up', 'do', 'ltgt', 'You', 'from', 'out', 'know', 'go', 'just', 'this', 'if', 'when', 'like']\n",
      "\n",
      "\n",
      "Go until jurong point crazy Available only in bugis n great world la e buffet Cine there got amore wat\n",
      "[813, 462, 5238, 918, 919, 2589, 62, 7, 1579, 83, 145, 593, 1389, 172, 3432, 5239, 61, 55, 5240, 180]\n",
      "\n",
      "\n",
      "102\n",
      "20\n"
     ]
    }
   ],
   "source": [
    "print (sorted_split_words[:50])\n",
    "print (\"\\n\")\n",
    "print (messages[0])\n",
    "print (messages_ints[0])\n",
    "print (\"\\n\")\n",
    "print (len(messages[0]))\n",
    "print (len(messages_ints[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Converting labels to 0 and 1 - SPAM:1 and NOT SPAM:0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "labels = labels.split(\"\\n\")\n",
    "labels = np.array([0 if label == \"ham\" else 1 for label in labels])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 1, 0, 0, 1, 0, 0, 1, 1])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Zero-length messages: 3\n",
      "Maximum message length: 171\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "\n",
    "message_lens = Counter([len(x) for x in messages_ints])\n",
    "print(\"Zero-length messages: {}\".format(message_lens[0]))\n",
    "print(\"Maximum message length: {}\".format(max(message_lens)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "messages_ints = [message for message in messages_ints if (len(message)>0)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Padding vectors with zeros so that all inputs are of same length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "seq_len = 200\n",
    "num_messages = len(messages)\n",
    "features = np.zeros([num_messages, seq_len], dtype=int)\n",
    "for i, row in enumerate(messages_ints):\n",
    "    features[i, -len(row):] = np.array(row)[:seq_len]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
       "          0,    0,    0,    0,  813,  462, 5238,  918,  919, 2589,   62,\n",
       "          7, 1579,   83,  145,  593, 1389,  172, 3432, 5239,   61,   55,\n",
       "       5240,  180])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Splitting into training, validation and test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\t\tFeature Shapes:\n",
      "Train set: \t\t(4460, 200) \n",
      "Validation set: \t(557, 200) \n",
      "Test set: \t\t(558, 200)\n",
      "\t\t\\Label Shapes:\n",
      "Train set: \t\t(4460,) \n",
      "Validation set: \t(557,) \n",
      "Test set: \t\t(556,)\n"
     ]
    }
   ],
   "source": [
    "split_frac1 = 0.8\n",
    "\n",
    "idx1 = int(len(features) * split_frac1)\n",
    "train_x, val_x = features[:idx1], features[idx1:]\n",
    "train_y, val_y = labels[:idx1], labels[idx1:]\n",
    "\n",
    "split_frac2 = 0.5\n",
    "idx2 = int(len(val_x) * split_frac2)\n",
    "val_x, test_x = val_x[:idx2], val_x[idx2:]\n",
    "val_y, test_y = val_y[:idx2], val_y[idx2:]\n",
    "\n",
    "print(\"\\t\\t\\tFeature Shapes:\")\n",
    "print(\"Train set: \\t\\t{}\".format(train_x.shape), \n",
    "      \"\\nValidation set: \\t{}\".format(val_x.shape),\n",
    "      \"\\nTest set: \\t\\t{}\".format(test_x.shape))\n",
    "\n",
    "print(\"\\t\\t\\Label Shapes:\")\n",
    "print(\"Train set: \\t\\t{}\".format(train_y.shape), \n",
    "      \"\\nValidation set: \\t{}\".format(val_y.shape),\n",
    "      \"\\nTest set: \\t\\t{}\".format(test_y.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initial prediction using Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.861759425494\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "clf = LogisticRegression()\n",
    "clf.fit(train_x,train_y)\n",
    "p = clf.predict(val_x)\n",
    "print (accuracy_score(val_y,p))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "lstm_size = 256\n",
    "lstm_layers = 1\n",
    "batch_size = 256\n",
    "learning_rate = 0.003"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating placeholder for inputs, labels and dropout rate "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "n_words = len(sorted_split_words)\n",
    "\n",
    "# Create the graph object\n",
    "graph = tf.Graph()\n",
    "# Add nodes to the graph\n",
    "with graph.as_default():\n",
    "    inputs_ = tf.placeholder(tf.int32, [None,None], name = \"inputs\")\n",
    "    labels_ = tf.placeholder(tf.int32, [None,None], name = \"labels\")\n",
    "    keep_prob = tf.placeholder(tf.float32, name = \"keep_prob\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Adding an embedding layer. Instead of one-hot encoding, we build an embedding layer and use that layer as a lookup table."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Size of the embedding vectors (number of units in the embedding layer)\n",
    "embed_size = 300 \n",
    "\n",
    "with graph.as_default():\n",
    "    embedding = tf.Variable(tf.random_uniform((n_words, embed_size), -1, 1))\n",
    "    embed = tf.nn.embedding_lookup(embedding, inputs_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create LSTM cells to use in the recurrent network  Here we are just defining what the cells look like.\n",
    "\n",
    "It takes a parameter called num_units, the number of units in the cell, called lstm_size in this code.\n",
    "\n",
    "Adding dropout to the cell with tf.contrib.rnn.DropoutWrapper. This wraps the cell in another cell, but with dropout added to the inputs and/or outputs. \n",
    "\n",
    "Here, [drop] * lstm_layers creates a list of cells (drop) that is lstm_layers long. The MultiRNNCell wrapper builds this into multiple layers of RNN cells, one for each cell in the list.\n",
    "\n",
    "So the final cell in the network is actually multiple (or just one) LSTM cells with dropout."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with graph.as_default():\n",
    "    # Your basic LSTM cell\n",
    "    lstm = tf.contrib.rnn.BasicLSTMCell(lstm_size)\n",
    "    \n",
    "    # Add dropout to the cell\n",
    "    drop = tf.contrib.rnn.DropoutWrapper(lstm, output_keep_prob=keep_prob)\n",
    "    \n",
    "    # Stack up multiple LSTM layers, for deep learning\n",
    "    cell = tf.contrib.rnn.MultiRNNCell([drop] * lstm_layers)\n",
    "    \n",
    "    # Getting an initial state of all zeros\n",
    "    initial_state = cell.zero_state(batch_size, tf.float32)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we need to actually run the data through the RNN nodes. You can use tf.nn.dynamic_rnn to do this. You'd pass in the RNN cell you created (our multiple layered LSTM cell for instance), and the inputs to the network.\n",
    "\n",
    "Initial_state is the cell state that is passed between the hidden layers in successive time steps. We pass in our cell and the input to the cell, then it does the unrolling and everything else for us. It returns outputs for each time step and the final_state of the hidden layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with graph.as_default():\n",
    "    outputs, final_state = tf.nn.dynamic_rnn(cell, embed, initial_state=initial_state)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We only care about the final output, we'll be using that as our sentiment prediction. So we need to grab the last output with outputs[:, -1], the calculate the cost from that and labels_."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "with graph.as_default():\n",
    "    predictions = tf.contrib.layers.fully_connected(outputs[:, -1], 1, activation_fn=tf.sigmoid)\n",
    "    cost = tf.losses.mean_squared_error(labels_, predictions)\n",
    "    \n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculating predictions and accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with graph.as_default():\n",
    "    correct_pred = tf.equal(tf.cast(tf.round(predictions), tf.int32), labels_)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating Batches"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_batches(x, y, batch_size=100):\n",
    "    \n",
    "    n_batches = len(x)//batch_size\n",
    "    x, y = x[:n_batches*batch_size], y[:n_batches*batch_size]\n",
    "    for ii in range(0, len(x), batch_size):\n",
    "        yield x[ii:ii+batch_size], y[ii:ii+batch_size]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0/5 Iteration: 5 Train loss: 0.135\n",
      "Epoch: 0/5 Iteration: 10 Train loss: 0.108\n",
      "Epoch: 0/5 Iteration: 15 Train loss: 0.099\n",
      "Epoch: 1/5 Iteration: 20 Train loss: 0.132\n",
      "Epoch: 1/5 Iteration: 25 Train loss: 0.099\n",
      "Val acc: 0.867\n",
      "Epoch: 1/5 Iteration: 30 Train loss: 0.091\n",
      "Epoch: 2/5 Iteration: 35 Train loss: 0.107\n",
      "Epoch: 2/5 Iteration: 40 Train loss: 0.097\n",
      "Epoch: 2/5 Iteration: 45 Train loss: 0.104\n",
      "Epoch: 2/5 Iteration: 50 Train loss: 0.123\n",
      "Val acc: 0.863\n",
      "Epoch: 3/5 Iteration: 55 Train loss: 0.098\n",
      "Epoch: 3/5 Iteration: 60 Train loss: 0.088\n",
      "Epoch: 3/5 Iteration: 65 Train loss: 0.085\n",
      "Epoch: 4/5 Iteration: 70 Train loss: 0.059\n",
      "Epoch: 4/5 Iteration: 75 Train loss: 0.060\n",
      "Val acc: 0.816\n",
      "Epoch: 4/5 Iteration: 80 Train loss: 0.051\n",
      "Epoch: 4/5 Iteration: 85 Train loss: 0.068\n"
     ]
    }
   ],
   "source": [
    "epochs = 5\n",
    "\n",
    "with graph.as_default():\n",
    "    saver = tf.train.Saver()\n",
    "\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    iteration = 1\n",
    "    for e in range(epochs):\n",
    "        state = sess.run(initial_state)\n",
    "        \n",
    "        for ii, (x, y) in enumerate(get_batches(train_x, train_y, batch_size), 1):\n",
    "            feed = {inputs_: x,\n",
    "                    labels_: y[:, None],\n",
    "                    keep_prob: 0.5,\n",
    "                    initial_state: state}\n",
    "            loss, state, _ = sess.run([cost, final_state, optimizer], feed_dict=feed)\n",
    "            \n",
    "            if iteration%5==0:\n",
    "                print(\"Epoch: {}/{}\".format(e, epochs),\n",
    "                      \"Iteration: {}\".format(iteration),\n",
    "                      \"Train loss: {:.3f}\".format(loss))\n",
    "\n",
    "            if iteration%25==0:\n",
    "                val_acc = []\n",
    "                val_state = sess.run(cell.zero_state(batch_size, tf.float32))\n",
    "                for x, y in get_batches(val_x, val_y, batch_size):\n",
    "                    feed = {inputs_: x,\n",
    "                            labels_: y[:, None],\n",
    "                            keep_prob: 1,\n",
    "                            initial_state: val_state}\n",
    "                    batch_acc, val_state = sess.run([accuracy, final_state], feed_dict=feed)\n",
    "                    val_acc.append(batch_acc)\n",
    "                print(\"Val acc: {:.3f}\".format(np.mean(val_acc)))\n",
    "            iteration +=1\n",
    "    saver.save(sess, \"checkpoints/sentiment.ckpt\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Testing "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from checkpoints/sentiment.ckpt\n",
      "Test accuracy: 0.789\n"
     ]
    }
   ],
   "source": [
    "test_acc = []\n",
    "with tf.Session(graph=graph) as sess:\n",
    "    saver.restore(sess, tf.train.latest_checkpoint('checkpoints'))\n",
    "    test_state = sess.run(cell.zero_state(batch_size, tf.float32))\n",
    "    for ii, (x, y) in enumerate(get_batches(test_x, test_y, batch_size), 1):\n",
    "        feed = {inputs_: x,\n",
    "                labels_: y[:, None],\n",
    "                keep_prob: 1,\n",
    "                initial_state: test_state}\n",
    "        batch_acc, test_state = sess.run([accuracy, final_state], feed_dict=feed)\n",
    "        test_acc.append(batch_acc)\n",
    "    print(\"Test accuracy: {:.3f}\".format(np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "text = 'Hello You just won yourself a free tour to Bahamas!! Call Now to recieve it.'\n",
    "text = ''.join([c for c in text if c not in punctuation])\n",
    "integer = ([vocab_to_int[i] for i in text.split(\" \")])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "i = []\n",
    "integer = np.array(integer)\n",
    "i.append(integer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1, 15)"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i = np.array(i)\n",
    "i.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./checkpoints/sentiment.ckpt\n"
     ]
    },
    {
     "ename": "InvalidArgumentError",
     "evalue": "ConcatOp : Dimensions of inputs should match: shape[0] = [1,300] vs. shape[1] = [256,256]\n\t [[Node: rnn/while/multi_rnn_cell/cell_0/basic_lstm_cell/basic_lstm_cell_1/concat = ConcatV2[N=2, T=DT_FLOAT, Tidx=DT_INT32, _device=\"/job:localhost/replica:0/task:0/cpu:0\"](rnn/while/TensorArrayReadV3, rnn/while/Identity_3, rnn/while/multi_rnn_cell/cell_0/basic_lstm_cell/basic_lstm_cell_1/concat/axis)]]\n\nCaused by op 'rnn/while/multi_rnn_cell/cell_0/basic_lstm_cell/basic_lstm_cell_1/concat', defined at:\n  File \"/anaconda/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/anaconda/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/anaconda/lib/python3.6/site-packages/ipykernel/__main__.py\", line 3, in <module>\n    app.launch_new_instance()\n  File \"/anaconda/lib/python3.6/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/anaconda/lib/python3.6/site-packages/ipykernel/kernelapp.py\", line 474, in start\n    ioloop.IOLoop.instance().start()\n  File \"/anaconda/lib/python3.6/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/anaconda/lib/python3.6/site-packages/tornado/ioloop.py\", line 887, in start\n    handler_func(fd_obj, events)\n  File \"/anaconda/lib/python3.6/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/anaconda/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/anaconda/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/anaconda/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/anaconda/lib/python3.6/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/anaconda/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 276, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/anaconda/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 228, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/anaconda/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 390, in execute_request\n    user_expressions, allow_stdin)\n  File \"/anaconda/lib/python3.6/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/anaconda/lib/python3.6/site-packages/ipykernel/zmqshell.py\", line 501, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/anaconda/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/anaconda/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/anaconda/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-34-13b4528366d0>\", line 2, in <module>\n    outputs, final_state = tf.nn.dynamic_rnn(cell, embed, initial_state=initial_state)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py\", line 553, in dynamic_rnn\n    dtype=dtype)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py\", line 720, in _dynamic_rnn_loop\n    swap_memory=swap_memory)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2623, in while_loop\n    result = context.BuildLoop(cond, body, loop_vars, shape_invariants)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2456, in BuildLoop\n    pred, body, original_loop_vars, loop_vars, shape_invariants)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2406, in _BuildLoop\n    body_result = body(*packed_vars_for_body)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py\", line 705, in _time_step\n    (output, new_state) = call_cell()\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py\", line 691, in <lambda>\n    call_cell = lambda: cell(input_t, state)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py\", line 953, in __call__\n    cur_inp, new_state = cell(cur_inp, cur_state)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py\", line 713, in __call__\n    output, new_state = self._cell(inputs, state, scope)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py\", line 241, in __call__\n    concat = _linear([inputs, h], 4 * self._num_units, True)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py\", line 1048, in _linear\n    res = math_ops.matmul(array_ops.concat(args, 1), weights)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py\", line 1034, in concat\n    name=name)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/gen_array_ops.py\", line 519, in _concat_v2\n    name=name)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py\", line 768, in apply_op\n    op_def=op_def)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 2336, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 1228, in __init__\n    self._traceback = _extract_stack()\n\nInvalidArgumentError (see above for traceback): ConcatOp : Dimensions of inputs should match: shape[0] = [1,300] vs. shape[1] = [256,256]\n\t [[Node: rnn/while/multi_rnn_cell/cell_0/basic_lstm_cell/basic_lstm_cell_1/concat = ConcatV2[N=2, T=DT_FLOAT, Tidx=DT_INT32, _device=\"/job:localhost/replica:0/task:0/cpu:0\"](rnn/while/TensorArrayReadV3, rnn/while/Identity_3, rnn/while/multi_rnn_cell/cell_0/basic_lstm_cell/basic_lstm_cell_1/concat/axis)]]\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m/anaconda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1038\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1039\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1040\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1020\u001b[0m                                  \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1021\u001b[0;31m                                  status, run_metadata)\n\u001b[0m\u001b[1;32m   1022\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/lib/python3.6/contextlib.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, type, value, traceback)\u001b[0m\n\u001b[1;32m     88\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 89\u001b[0;31m                 \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgen\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     90\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mStopIteration\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py\u001b[0m in \u001b[0;36mraise_exception_on_not_ok_status\u001b[0;34m()\u001b[0m\n\u001b[1;32m    465\u001b[0m           \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpywrap_tensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 466\u001b[0;31m           pywrap_tensorflow.TF_GetCode(status))\n\u001b[0m\u001b[1;32m    467\u001b[0m   \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: ConcatOp : Dimensions of inputs should match: shape[0] = [1,300] vs. shape[1] = [256,256]\n\t [[Node: rnn/while/multi_rnn_cell/cell_0/basic_lstm_cell/basic_lstm_cell_1/concat = ConcatV2[N=2, T=DT_FLOAT, Tidx=DT_INT32, _device=\"/job:localhost/replica:0/task:0/cpu:0\"](rnn/while/TensorArrayReadV3, rnn/while/Identity_3, rnn/while/multi_rnn_cell/cell_0/basic_lstm_cell/basic_lstm_cell_1/concat/axis)]]",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m                      Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-105-c36609a40c9e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m                  \u001b[0minitial_state\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mstate\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m                  keep_prob: 0.5}\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0mpredictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Test accuracy: {:.3f}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    776\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    777\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 778\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    779\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    780\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    980\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    981\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m--> 982\u001b[0;31m                              feed_dict_string, options, run_metadata)\n\u001b[0m\u001b[1;32m    983\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    984\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1030\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1031\u001b[0m       return self._do_call(_run_fn, self._session, feed_dict, fetch_list,\n\u001b[0;32m-> 1032\u001b[0;31m                            target_list, options, run_metadata)\n\u001b[0m\u001b[1;32m   1033\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1034\u001b[0m       return self._do_call(_prun_fn, self._session, handle, feed_dict,\n",
      "\u001b[0;32m/anaconda/lib/python3.6/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1050\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1051\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1052\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1053\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1054\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mInvalidArgumentError\u001b[0m: ConcatOp : Dimensions of inputs should match: shape[0] = [1,300] vs. shape[1] = [256,256]\n\t [[Node: rnn/while/multi_rnn_cell/cell_0/basic_lstm_cell/basic_lstm_cell_1/concat = ConcatV2[N=2, T=DT_FLOAT, Tidx=DT_INT32, _device=\"/job:localhost/replica:0/task:0/cpu:0\"](rnn/while/TensorArrayReadV3, rnn/while/Identity_3, rnn/while/multi_rnn_cell/cell_0/basic_lstm_cell/basic_lstm_cell_1/concat/axis)]]\n\nCaused by op 'rnn/while/multi_rnn_cell/cell_0/basic_lstm_cell/basic_lstm_cell_1/concat', defined at:\n  File \"/anaconda/lib/python3.6/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/anaconda/lib/python3.6/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/anaconda/lib/python3.6/site-packages/ipykernel/__main__.py\", line 3, in <module>\n    app.launch_new_instance()\n  File \"/anaconda/lib/python3.6/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/anaconda/lib/python3.6/site-packages/ipykernel/kernelapp.py\", line 474, in start\n    ioloop.IOLoop.instance().start()\n  File \"/anaconda/lib/python3.6/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/anaconda/lib/python3.6/site-packages/tornado/ioloop.py\", line 887, in start\n    handler_func(fd_obj, events)\n  File \"/anaconda/lib/python3.6/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/anaconda/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/anaconda/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/anaconda/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/anaconda/lib/python3.6/site-packages/tornado/stack_context.py\", line 275, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/anaconda/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 276, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/anaconda/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 228, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/anaconda/lib/python3.6/site-packages/ipykernel/kernelbase.py\", line 390, in execute_request\n    user_expressions, allow_stdin)\n  File \"/anaconda/lib/python3.6/site-packages/ipykernel/ipkernel.py\", line 196, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/anaconda/lib/python3.6/site-packages/ipykernel/zmqshell.py\", line 501, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/anaconda/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2717, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/anaconda/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2821, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/anaconda/lib/python3.6/site-packages/IPython/core/interactiveshell.py\", line 2881, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-34-13b4528366d0>\", line 2, in <module>\n    outputs, final_state = tf.nn.dynamic_rnn(cell, embed, initial_state=initial_state)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py\", line 553, in dynamic_rnn\n    dtype=dtype)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py\", line 720, in _dynamic_rnn_loop\n    swap_memory=swap_memory)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2623, in while_loop\n    result = context.BuildLoop(cond, body, loop_vars, shape_invariants)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2456, in BuildLoop\n    pred, body, original_loop_vars, loop_vars, shape_invariants)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2406, in _BuildLoop\n    body_result = body(*packed_vars_for_body)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py\", line 705, in _time_step\n    (output, new_state) = call_cell()\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py\", line 691, in <lambda>\n    call_cell = lambda: cell(input_t, state)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py\", line 953, in __call__\n    cur_inp, new_state = cell(cur_inp, cur_state)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py\", line 713, in __call__\n    output, new_state = self._cell(inputs, state, scope)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py\", line 241, in __call__\n    concat = _linear([inputs, h], 4 * self._num_units, True)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/contrib/rnn/python/ops/core_rnn_cell_impl.py\", line 1048, in _linear\n    res = math_ops.matmul(array_ops.concat(args, 1), weights)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/array_ops.py\", line 1034, in concat\n    name=name)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/ops/gen_array_ops.py\", line 519, in _concat_v2\n    name=name)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py\", line 768, in apply_op\n    op_def=op_def)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 2336, in create_op\n    original_op=self._default_original_op, op_def=op_def)\n  File \"/anaconda/lib/python3.6/site-packages/tensorflow/python/framework/ops.py\", line 1228, in __init__\n    self._traceback = _extract_stack()\n\nInvalidArgumentError (see above for traceback): ConcatOp : Dimensions of inputs should match: shape[0] = [1,300] vs. shape[1] = [256,256]\n\t [[Node: rnn/while/multi_rnn_cell/cell_0/basic_lstm_cell/basic_lstm_cell_1/concat = ConcatV2[N=2, T=DT_FLOAT, Tidx=DT_INT32, _device=\"/job:localhost/replica:0/task:0/cpu:0\"](rnn/while/TensorArrayReadV3, rnn/while/Identity_3, rnn/while/multi_rnn_cell/cell_0/basic_lstm_cell/basic_lstm_cell_1/concat/axis)]]\n"
     ]
    }
   ],
   "source": [
    "with tf.Session(graph=graph) as sess:\n",
    "    ckpt = tf.train.get_checkpoint_state('./checkpoints')\n",
    "    state = sess.run(initial_state)\n",
    "    saver.restore(sess, ckpt.model_checkpoint_path)\n",
    "    feed_dict = {inputs_: i, \n",
    "                 initial_state: state,\n",
    "                 keep_prob: 0.5}\n",
    "    predictions = sess.run(predictions, feed_dict = feed_dict)\n",
    "\n",
    "print(\"Test accuracy: {:.3f}\".format(predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
